#!/usr/bin/perl
#################################
# Metadata Enhancement for Repositories (MER) 
# This software is an overlay repository search interface that uses a 
# text mined index of terms.
# Copyright (C) 2011  Rory McNicholl (University of London Computer Centre), Robert Drinkall (University College London)

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

# Contact repositories@ulcc.ac.uk
# http://www.ulcc.ac.uk

##################################

## strip_pdf makes an attempt at stripping out headers footers, columns and tables from pdfs
## It does this by converting the pdf to xml (using pdftohtml) and traversing the DOM making 
## decisions based on position and frequency of text.

## If the first or last bit of text on each page is the same... 
## then we are probably looking at a header or footer, that sort of thing.

## Rory McNicholl
use strict;
use XML::LibXML;

my $pdf = $ARGV[0];
unless($pdf){ 
	print "Usage: strip_pdf input_pdf [threshold]\n";
	exit;
}
my $threshold = $ARGV[1] || 80;

if(!-e $pdf){
	die("$pdf not found");
}

my $H_and_F = 1;
my $References = 1;
my $Tables = 1;
my $as_xml = 1;

my $pdftohtml = "/usr/local/bin/pdftohtml -i -xml $pdf 1> /dev/null 2> /tmp/strip_pdf_err";
#print $pdftohtml."\n";
system($pdftohtml) == 0 or die("could not run pdftohtml: $!");

if(-s "/tmp/strip_pdf_err"){
	open(FILE, "/tmp/strip_pdf_err");
	while(<FILE>){
		if(/Copying of text from this document is not allowed/){
			print STDERR "Have error converting pdf to html trying conversion to text and stripping references\n";
			$H_and_F = 0;
			$Tables = 0;
			$as_xml = 0;
			last;
		}
	}
}

if(!$as_xml){
	my $pdftotext = "/usr/bin/pdftotext -layout $pdf 1> /dev/null 2> /tmp/stripguff_err";
	system($pdftotext) == 0 or die("could not run pdftotext: $!");

	my $txt = $pdf;
	$txt =~ s/\.pdf$/\.txt/;

	my ($text, $references);

	if(-e $txt){ 
		open(FILE, "$txt");
		my ($prev, $next);
		my $is_ref=0;
		my $stopped = 0;
		while(<FILE>){
			if($is_ref){
				if(/\s|/ && $prev =~ /\s|/ && $stopped == 0){
					print STDERR "Found 'References' title stopping here...\n";
					$stopped = 1;
				}else{
					$is_ref = 0;
				}
			}
			if($stopped){
				$references .= $_;
			}else{
				$text.=$_;
			}

			if(/^\s*References\s*$/){
				$is_ref = 1;
			}else{
				$prev = $_;
			}
			
		}
		close(FILE);

	}else{ 
		print STDERR "No text file found, exiting\n";
	}
	print STDOUT $text;

}else{

	# Open xml and create LibXML object
	my $xml = $pdf;
	if($xml =~ /\.pdf$/){
		$xml =~ s/\.pdf$/\.xml/;
	}else{
		$xml .= ".xml";
	}
	my $xml_str;
	
	# remove odd chars...
	open(FILE, "$xml");
	while(<FILE>){
		s/[^[:ascii:]]+//g;
		unless(/\cP/){
			$xml_str.=$_;
		}
	}

	my $parser = XML::LibXML->new();
#	my $doc = $parser->parse_file($xml);
	my $doc = $parser->parse_string($xml_str);

	my $pdf_stats = {};

	my $query  = "//pdf2xml/page";
	my @pages = $doc->findnodes($query);
	my $num_pages = scalar @pages;
	print STDERR "Processing $num_pages pages\n";
	my $ref_flag = 0;
	# Loop through pages
	foreach my $page (@pages){
		my ($first,@everything, $first_lines); #Thanks Barry
		my $f_stop=0;
		my $para;
		my $paras = ();
		my $para_str="";
		my $query  = "text/text() | text/b/text() | text/i/text()";
		my @lines = $page->findnodes($query);
		# Loop through lines on pages
		for( my $n=0; $n<= $#lines; $n++){
			$_ = $lines[$n];
#			print $_->data."\n";
			# React to blank lines to identify paragraphs and headings
			if($_->data =~ /^\s*$/ || $n == $#lines){
				$f_stop++;
				if($para_str !~ /^\s*$/){
					push @everything, {string => $para_str, lines => $para};
					$para = ();
					$para_str = "";
				}
			}else{
				push @{$para}, $_;
				$para_str .= $_->data;
			}
			# Tag the first paragraph of each page
			if($f_stop<2){
				$first .= $_->data;
				push @{$first_lines}, $_->data;
				$_->parentNode->setAttribute("class", "first_para");
			}
		}
		# Check for references
		foreach my $p (@everything){
			foreach my $l (@{$p->{lines}}){
				if($l->data =~ /^\s*References\s*$/){
					$l->parentNode->setAttribute("rel", "references");
					$ref_flag = 1;
				}
				if($ref_flag){
					$l->parentNode->setAttribute("rel", "references");
					print STDERR "tagging line as reference\n";
				}
			}
		}

		# Collect stats for first paragraphs
		$pdf_stats->{first}->{$first}->{count}++;
		$pdf_stats->{first}->{$first}->{lines} = $first_lines;
	
		# Tag the last paragraph
		my $last = pop(@everything);
		my $last_lines;
		foreach my $l (@{$last->{lines}}){
			$l->parentNode->setAttribute("class", "last_para");
			push @{$last_lines}, $l->data;
		}
		# Collect stats for last paragraphs
		$pdf_stats->{"last"}->{$last->{string}}->{count}++;
		$pdf_stats->{"last"}->{$last->{string}}->{lines} = $last_lines;
	
	}

	########################
	# Assess headers
	########################
	my $header_mess = "No likely header found";
	my $header;
	while(my($first,$data)=each(%{$pdf_stats->{first}})){
		my $pc = $data->{count}/$num_pages*100;
		if($num_pages gt 1 && $pc > $threshold){
			$header_mess = "Probable header: \"$first\" (".sprintf('%.2f',$pc)." %)";
			$header = $data->{lines};
		}
	}
	########################
	# Remove tagged headers
	########################
	my $query  = "//pdf2xml/page/text[\@class = 'first_para']/text() | //pdf2xml/page/text/b[\@class = 'first_para']/text() | //pdf2xml/page/text/i[\@class = 'first_para']/text()";
	foreach ($doc->findnodes($query)){
		my $header_line = $_;
		#it will only remove a first paragraph if it has been tagged as a header by virtue of > threshold frequency
		if(grep $_ eq $header_line->data, @$header){ 
			my $page = $header_line->parentNode;
			$page->removeChild($header_line);
		}
	}
	
	########################
	# Assess footers
	########################
	my $footer_mess = "No likely footer found";
	my $footer;
	while(my($last,$data)=each(%{$pdf_stats->{"last"}})){
		my $pc = $data->{count}/$num_pages*100;
		if($num_pages gt 1 && $pc > $threshold){
			$footer_mess = "Probable header: \"$last\" (".sprintf('%.2f',$pc)." %)";
			$footer = $data->{lines};
		}
	}
	########################
	# Remove tagged footers
	########################
	my $query  = "//pdf2xml/page/text[\@class = 'last_para']/text() | //pdf2xml/page/text/b[\@class = 'last_para']/text() | //pdf2xml/page/text/i[\@class = 'last_para']/text()";
	foreach ($doc->findnodes($query)){
		my $footer_line = $_;
		#it will only remove a first paragraph if it has been tagged as a header by virtue of > threshold frequency
		if(grep $_ eq $footer_line->data, @$footer){ 
			my $page = $footer_line->parentNode;
			print STDERR "removing footer ".$footer_line->data."\n";
			$page->removeChild($footer_line);
		}
	}
	########################
	# Remove tagged references
	########################
	my $query = "//pdf2xml/page/text[\@rel = 'references']/text() | //pdf2xml/page/text/b[\@rel = 'references']/text() | //pdf2xml/page/text/i[\@rel = 'references']/text()";
	my $ref_count = 0;
	foreach ($doc->findnodes($query)){
		my $page = $_->parentNode;
		$page->removeChild($_);
		$ref_count++;
	}
	my $references_mess =  "Removed $ref_count reference lines\n";

	########################
	# Report
	########################
	print STDERR $footer_mess."\n";
	print STDERR $header_mess."\n";
	print STDERR $references_mess."\n";

	########################
	# Output remaining text
	########################
	my $query  = "//pdf2xml/page/text/text()";
	my @lines = $doc->findnodes($query);
	binmode STDOUT, ":utf8"; 
	foreach my $line (@lines){
		print STDOUT $line->data."\n";
	}
}
=comment
	sub get_end_node {
	    my( $node ) = @_;
	    unless( $node->nodeType eq &XML_ELEMENT_NODE ) { return $node };
	    foreach my $child ( $node->getChildnodes ) {
	        &get_end_node( $child );
	    }
	}
=cut
